#!/usr/bin/python

"""
read a freemap rdb file, and return a dictionary of parsed tables.
currently only some of the tables are read.
"""

import os, sys
from struct import unpack, calcsize


class Struct(dict):
	def __getattr__(self,name):
		try:
			val=self[name]
		except KeyError:
			val=super(Struct,self).getattr(self,name)
		return val
	def __setattr__(self,name,val):
		l=dir(self)
		if name in self:
			super(Struct,self).setattr(self,name,val)
		else:
			self[name]=val


# strip everything after (and including) the first null byte
def asciiz(string):
    null_index = string.index('\x00');
    return string[0:null_index]

###############################################################################
## basic rdb file read functions (table level)

def roadmap_get_section_header(parent):
    #struct roadmap_db_section {
    #   char name[12];
    #   int  first;     // first child
    #   int  next;      // next sibling section
    #   int  size;      // size of the this section (not including roadmap_db_section header
    #   int  count;  /* .. of records, when applicable. */
    #};
    fmt_str = "12s4i"
    (_name, _first, _next, _size, _count) = \
             unpack(fmt_str, rdb_bytes[ parent : parent+calcsize(fmt_str) ])
    return Struct(name=asciiz(_name), first=_first, next=_next, size=_size, count=_count)


#roadmap_db *roadmap_db_get_subsection (roadmap_db *parent, char *path);
def roadmap_db_get_subsection (parent, path):
  child = roadmap_db_get_first(parent)
  while child is not None:
    if roadmap_db_get_name(child)==path:
        return child
    child = roadmap_db_get_next(child)
  return None # not found

#roadmap_db *roadmap_db_get_first (roadmap_db *parent);
def roadmap_db_get_first (parent):
    first = roadmap_get_section_header(parent).first
    if first<>0:
        return first # + parent
    else:
        return None

#char       *roadmap_db_get_name  (roadmap_db *section);
def roadmap_db_get_name (parent):
    return roadmap_get_section_header(parent).name

#unsigned    roadmap_db_get_size  (roadmap_db *section);
def roadmap_db_get_size (parent):
    return roadmap_get_section_header(parent).size

#int         roadmap_db_get_count (roadmap_db *section);
def roadmap_db_get_count (parent):
    return roadmap_get_section_header(parent).count

#int         roadmap_db_get_level (roadmap_db *section);
# NOT IMPLEMENTED!

#char       *roadmap_db_get_data  (roadmap_db *section);
def roadmap_db_get_data(parent):
    return parent + 28  # skip sizeof( struct roadmap_db_section )

#roadmap_db *roadmap_db_get_next  (roadmap_db *section);
def roadmap_db_get_next(parent):
    next = roadmap_get_section_header(parent).next
    if next<>0:
        return next # + parent
    else:
        return None



# print a list of all sections in the rdb file
def dump_sections(root, level=""):
  print level, roadmap_db_get_name(root), \
        "\t [%d] ; %d" % (roadmap_db_get_count(root),roadmap_db_get_size(root))
  level = "%s  " % level
  child = roadmap_db_get_first(root)
  while child is not None:
    dump_sections(child, level)
    child = roadmap_db_get_next(child)

#dump_sections(0)

###############################################################################




###############################################################################
## read string tables

# write a single string.xxx talbe
# input: pointer to string.xxx table
def load_string_sub_table(string_sub_table):
  tbl_data   = roadmap_db_get_subsection (string_sub_table, "data");
  tbl_index  = roadmap_db_get_subsection (string_sub_table, "index");

  data_bytes = roadmap_db_get_data (tbl_data);  # char *data_bytes
  data_len = roadmap_db_get_count(tbl_data);      # int data_len

  index_bytes = roadmap_db_get_data (tbl_index); # unsigned int *index_bytes
  index_len = roadmap_db_get_count (tbl_index); # int index_len

  result = {}

  #print "string.xxx.data %d records, string.xxx.index %d records\n" % ( data_len, index_len );
  for i in range(0,index_len):
    #print "          %d:/%s/, \n"%(i, data_bytes+*index_bytes);  # C CODE
    str_start = data_bytes + unpack("i", rdb_bytes[index_bytes:index_bytes+4] )[0]
    str_end = rdb_bytes.index('\x00',str_start)
    result[i] = rdb_bytes[str_start:str_end]
    index_bytes = index_bytes+4;

  return result


def load_string_table(root_table):
  tbl_string = roadmap_db_get_subsection (root_table, "string")
  parent = tbl_string;
  result = {}
  child = roadmap_db_get_first(parent)
  while child is not None:
    table_name = roadmap_db_get_name(child)
    #print "  '%s': \n" % table_name
    result[table_name] = load_string_sub_table(child)
    child = roadmap_db_get_next(child)
  return result
###############################################################################

###############################################################################
## read point tables

#typedef struct {  /* table point.data */
#
#   /* The position here is relative to the upper-left corner
#    * of the square the point belongs to.
#    */
#  unsigned short longitude;
#  unsigned short latitude;
#
#} RoadMapPoint;

def load_point_table(root_table):
   point_root_table = roadmap_db_get_subsection (root_table, "point");
   #//roadmap_db *bysquare_table  = roadmap_db_get_subsection (root, "bysquare");
   point_table = roadmap_db_get_subsection (point_root_table, "data");
   id_table = roadmap_db_get_subsection (point_root_table, "id");

   result = [] #point_data_table
   if id_table is not None:
     bytes = roadmap_db_get_data (point_table); # char *bytes
     #RoadMapPoint* values = (RoadMapPoint*)bytes;
     count=roadmap_db_get_count (point_table); # int count
     struct_fmt ="HH" # unsigned short longitude,latitude
     struct_size = calcsize(struct_fmt)
     for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
        result.append(  { 'longitude':struct_val[0], 'latitude':struct_val[1] }  )
        bytes = bytes + struct_size
   #
   #point_id_table={}
   #if id_table is not None:
   #  values = roadmap_db_get_data (id_table); # int *values
   #  count=roadmap_db_get_count (id_table); # int count
   #  print "point_id_table:  %d records\n" %  count
   #  for i in range(0, count):
   #     (id,) = unpack("i", rdb_bytes[values:values+4])
   #     print id
   #     point_id_table[i]=id
   #     print "<point_id index='%d', value='%d' value_as_hex='%08X' />\n" % (i, id, id);
   #     values = values + 4
   #//if (roadmap_db_get_size(point_table) !=
   #//       context->PointCount * sizeof(RoadMapPoint)) 
   #//   roadmap_log (ROADMAP_FATAL, "invalid point/data structure");
   #
   return result
###############################################################################

###############################################################################
## read street tables

#typedef struct {
#   RoadMapString fedirp;   // RoadMapString is ushort
#   RoadMapString fename;
#   RoadMapString fetype;
#   RoadMapString fedirs;
##ifndef J2MEMAP
#   RoadMapString t2s;
##endif
#} RoadMapStreet;

def load_street_table(root_table):
      tbl_street = roadmap_db_get_subsection (root_table, "street");
      tbl_name = roadmap_db_get_subsection (tbl_street, "name");
      tbl_type = roadmap_db_get_subsection (tbl_street, "type");

      name_bytes = roadmap_db_get_data (tbl_name);
      count=roadmap_db_get_count (tbl_name) # int count
      type_values = roadmap_db_get_data (tbl_type); # char* type_values

      ## street_name_table: record = struct RoadMapStreet
      street_name_table = []
      struct_fmt ="5H" # unsigned short fedirp,fename,fetype,fedirs,t2s
      struct_size = calcsize(struct_fmt)
      for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[name_bytes:name_bytes+struct_size])
        tmp = { 'index_in_table':i, 'fedirp':struct_val[0], \
                'fename':struct_val[1], 'fetype':struct_val[2], \
                'fedirs':struct_val[3], 't2s':struct_val[4]}
        street_name_table.append(tmp)
        name_bytes = name_bytes + struct_size

      ## street_type_table: record = 1 char
      street_type_table = []
      count=roadmap_db_get_count (tbl_type);
      for i in range(0, count):
        street_type_table.append( unpack("B" ,rdb_bytes[type_values:type_values+1])[0] )
        type_values = type_values + 1;

      return (street_name_table, street_type_table)
###############################################################################

###############################################################################
## read square tables

#typedef struct {
#   RoadMapArea edges;
#   int position;
#   int first_point;
#   int first_shape;
#} RoadMapSquare;
#typedef struct {
#   RoadMapArea edges;
#   int step_longitude;
#   int step_latitude;
#   int count_longitude;
#   int count_latitude;
#   int count_squares;  /* Necessary because empty squares have been removed. */
#} RoadMapGlobal;
def load_square_table(root_table):
      tbl_square = roadmap_db_get_subsection (root_table, "square");

      ## square_global_table: record = struct RoadMapGlobal  (single record!!)
      tbl_global = roadmap_db_get_subsection (tbl_square, "global");
      bytes = roadmap_db_get_data (tbl_global);
      struct_fmt ="4i5i" # edges.east,edges.north,edges.west,edges.south,step_longitude,step_latitude,count_longitude,count_latitude,count_squares
      struct_size = calcsize(struct_fmt)
      struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
      square_global_table={ 
        'edges.east': struct_val[0],
        'edges.north': struct_val[1],
        'edges.west': struct_val[2],
        'edges.south': struct_val[3],
        'step_longitude': struct_val[4],
        'step_latitude': struct_val[5],
        'count_longitude': struct_val[6],
        'count_latitude': struct_val[7],
        'count_squares': struct_val[8]
      }

      ## square_data_table: record = struct RoadMapSquare
      tbl_data = roadmap_db_get_subsection (tbl_square, "data");
      bytes = roadmap_db_get_data (tbl_data);
      count = roadmap_db_get_count (tbl_data);
      square_data_table=[]
      struct_fmt ="4i3i" #  edges.east,edges.north,edges.west,edges.south,position,first_point,first_shape
      struct_size = calcsize(struct_fmt)
      #//-----------
      #if (roadmap_db_get_size(tbl_data) != roadmap_db_get_count (tbl_data) * sizeof(RoadMapSquare))
      #   roadmap_log (ROADMAP_FATAL, "invalid point/data structure");
      #//-----------
      for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
        temp =  {
          'edges.east': struct_val[0],
          'edges.north': struct_val[1],
          'edges.west': struct_val[2],
          'edges.south': struct_val[3],
          'position': struct_val[4],
          'first_point': struct_val[5],
          'first_shape': struct_val[6]
        }
        square_data_table.append(temp)
        bytes = bytes + struct_size

      return (square_global_table, square_data_table)
###############################################################################

###############################################################################
## read line tables

#typedef struct {  /* table line */
#   int from;
#   int to;
#   unsigned short first_shape;
#   unsigned short street;
#} RoadMapLine;
def load_line_table(root_table):
      tbl_line = roadmap_db_get_subsection (root_table, "line");
      tbl_data = roadmap_db_get_subsection (tbl_line, "data");
      bytes = roadmap_db_get_data (tbl_data);
      count = roadmap_db_get_count (tbl_data);

      ## line_data_table: record = struct RoadMapLine
      line_data_table = []
      struct_fmt ="iiHH" # from,to,first_shape,street
      struct_size = calcsize(struct_fmt)
      for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
        temp =  { 'line_pos_index':i,
                  'from':struct_val[0], 'to':struct_val[1],
                  'first_shape':struct_val[2], 'street':struct_val[3] }
        line_data_table.append(temp)
        bytes = bytes + struct_size

      return line_data_table
###############################################################################

###############################################################################
## read polygon tables

#typedef struct {  /* table polygon.head */
#   unsigned short first;0
#   unsigned short count;1
#   RoadMapString name;2
#   char  cfcc;3
#   unsigned char filler;
#   /* TBD: replace these with a RoadMapArea (not compatible!). */
#   int   north;4
#   int   west;5
#   int   east;6
#   int   south;7
#} RoadMapPolygon;
#
#typedef struct {  /* table polygon.lines */
#   int point;
#} RoadMapPolygonPoint;

def load_polygon_table(root_table):
      tbl_polygon = roadmap_db_get_subsection (root_table, "polygon");

      ## polygon_head_table: record = struct RoadMapPolygon
      tbl_head = roadmap_db_get_subsection (tbl_polygon, "head");
      bytes = roadmap_db_get_data (tbl_head);
      count  = roadmap_db_get_count(tbl_head ); # int count_head

      polygon_head_table = []
      struct_fmt ="HHHBB4i" # first,count,name,cfcc,filler,north,west,east,south
      struct_size = calcsize(struct_fmt)
      for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
        temp = { 'first':struct_val[0], 'count':struct_val[1], 'name':struct_val[2], 'cfcc':struct_val[3], 'filler':struct_val[4], \
                 'north':struct_val[4], 'west':struct_val[5], 'east':struct_val[6], 'south':struct_val[7] }
        polygon_head_table.append(temp)
        bytes = bytes + struct_size;


      ## polygon_point_table: record = struct RoadMapPolygonPoint
      tbl_point = roadmap_db_get_subsection (tbl_polygon, "point");
      bytes = roadmap_db_get_data (tbl_point);
      count = roadmap_db_get_count(tbl_point); # int count_point
      polygon_point_table = []
      struct_fmt ="i" # point
      struct_size = calcsize(struct_fmt)
      for i in range(0, count):
        struct_val = unpack(struct_fmt, rdb_bytes[bytes:bytes+struct_size])
        polygon_point_table.append( struct_val[0] )
        bytes = bytes + struct_size;

      return (polygon_head_table, polygon_point_table)
###############################################################################

# EXPORTED FUNCTION
# input: rdb file path
# prints all tables / sections in the rdb file
def rdb_dump_tables(filepath):
        rdb_file = open(filepath, 'rb')
        global rdb_bytes ## TEMP!!!
        rdb_bytes = rdb_file.read()  # GLOBAL VARIABLE!! used by roadmap_db_XXX
        rdb_file.close()
        dump_sections(0)

# EXPORTED FUNCTION
# input: rdb file path
# return: dictionary of tables
def rdb_load(filepath):
        rdb_file = open(filepath, 'rb')
        global rdb_bytes ## TEMP!!!
        rdb_bytes = rdb_file.read()  # GLOBAL VARIABLE!! used by roadmap_db_XXX
        rdb_file.close()
        #print "rdb file size: ", len(rdb_bytes)

        tables = {}

        string_table = load_string_table(0)
        tables["string"] = string_table
        
        tables["point"] = {}
        tables["point"]["data"] = load_point_table(0)

        tables["street"] = {}
        (tables["street"]["name"], tables["street"]["type"]) = load_street_table(0)
        #global street_name_table ## TEMP!!!
        #global street_type_table ## TEMP!!!
        #street_name_table = tables["street"]["name"]
        #street_type_table = tables["street"]["type"]

        tables["square"] = {}
        (tables["square"]["global"], tables["square"]["data"]) = load_square_table(0)
        #global square_global_table
        #global square_data_table
        #square_global_table = tables["square"]["global"]
        #square_data_table = tables["square"]["data"]

        tables["line"] = {}
        tables["line"]["data"] = load_line_table(0)
        #global line_data_table
        #line_data_table = tables["line"]["data"]

        tables["polygon"] = {}
        (tables["polygon"]["head"], tables["polygon"]["point"]) = load_polygon_table(0)
        #global polygon_head_table
        #global polygon_point_table
        #polygon_head_table = tables["polygon"]["head"]
        #polygon_point_table = tables["polygon"]["point"]

        rdb_bytes = None # temp. since rdb_bytes is global, need to free the mem
        return tables


# EXPORTED FUNCTION??
# param uint32 point_id:
#        point_id>>16  - square index (0..square_global_table['count_squares'])
#        point_id>>16  - point_index in square
# param tables: value returned from rdb_load()
# return: 32bit index_in_db_data_table
def point_id_to_index(point_id, tables):
    square_index = point_id >> 16
    point_index_in_square = point_id & 0xFFFF
    #print "point_id=%08x, square_index=0x%x, point_index_in_square=0x%x" % (point_id, square_index,point_index_in_square)
    return tables["square"]["data"][square_index]['first_point'] + point_index_in_square


# EXPORTED FUNCTION
# this function adds calculated fields to some of the tables. see code below
# to know what is added. all calculated fields start with "_".
# input: return value of rdb_load
# return: none
def rdb_partial_parse(tables):
	point_data_table  = tables["point"]["data"]
	square_data_table = tables["square"]["data"]
	line_data_table = tables["line"]["data"]
	street_name_table = tables["street"]["name"]
        string_table = tables["string"]

	# get all point_id values from line_data_table
	# eliminate duplicates using set
	line_data_table = tables["line"]["data"]
	all_point_id = map(lambda line: line['from'], line_data_table);
	all_point_id += map(lambda line: line['to'], line_data_table)
	all_point_id = set(all_point_id)

	if len(all_point_id) <> len(point_data_table):
	    print >> sys.stderr, "error: len(all_point_id) <> len(point_data_table)"
	    print >> sys.stderr, "       not all points from point_data_table appear in line_data_table"
	    sys.exit(-1)

	# add abs_lon, abs_lat to point_data_table 
	for point_id in all_point_id:
	    square_index = point_id >> 16
	    point_index = point_id_to_index(point_id, tables)
	    point_data_table[point_index]['_abs_lon']= \
		     (point_data_table[point_index]['longitude']+ \
		      square_data_table[square_index]['edges.west']) / 1000000.0
	    point_data_table[point_index]['_abs_lat']= \
		     (point_data_table[point_index]['latitude']+ \
		      square_data_table[square_index]['edges.south']) / 1000000.0

	# add street name to street_name_table (and line_data_table??)
	for i in range(0, len(line_data_table)):
	    line = line_data_table[i]
            # _to, _from are indexes into tables[point][data] table
            line['_from'] = point_id_to_index(line['from'], tables)
            line['_to'] = point_id_to_index(line['to'], tables)
            street_name_str = ""
	    if line['street']<>65535:
		street_name=street_name_table[line['street']]
		if street_name['fedirp']<>65535:
		    street_name_str += string_table['prefix'][street_name['fedirp']] + " "
		if street_name['fename']<>65535:
		    street_name_str += string_table['street'][street_name['fename']]
		if street_name['fedirs']<>65535:
		    street_name_str += " " + string_table['suffix'][street_name['fedirs']]
            line['_name'] = street_name_str


# example:        
#RDB_FILE_NAME = "usc77001.rdm.original.2008-04-24"
#RDB_FILE_NAME = "usc77001.rdm.original.2008-05-22"  # newer
#tables = rdb_load(RDB_FILE_NAME)
#rdb_partial_parse(tables)

###############################################################################
## end of rdb reading section
###############################################################################



